# -*- coding: utf-8 -*-
import scrapy
from urllib import parse
import urllib
import re

frist = input("请输入职位")
frist = urllib.parse.quote(frist)

class JobsSpider(scrapy.Spider):
    name = 'jobs'
    allowed_domains = ['51job.com']
    start_urls = ['https://search.51job.com/list/030200%252C040000,000000,0000,00,9,99,'+str(frist)+',2,1.html?']

    def parse(self, response):
        #抓取职位数量
        SL = response.xpath('//*[@id="resultList"]/div[2]/div[4]/text()').get(default='')
        SL = re.findall('共(.*?)条职位',SL)[0]
        #根据数量确定页数
        if int(SL) % 50 != 0:
            page = int(SL) // 50 + 1
            print(page)
        else:
            page = int(SL) // 50
            print(page)
        #通过修改url来爬取所有页面
        for i in range(1,page+1):
            url = 'https://search.51job.com/list/030200%252C040000,000000,0000,00,9,99,'+str(frist)+',2,{}.html?'.format(i)
            #print(url)
            yield scrapy.Request(url,callback=self.parseDetail)

    def parseDetail(self,response):
        #爬取职位详情页面的url
        URL_list = response.xpath('//*[@id="resultList"]/div/p/span/a/@href').extract()
        for URL in URL_list:
            yield scrapy.Request(URL,callback=self.parseXX)

    def parseXX(self,response):
        #爬取详情信息
        ZW = response.xpath('/html/body/div[3]/div[2]/div[2]/div/div[1]/h1/text()').get(default='')
        GZ = response.xpath('/html/body/div[3]/div[2]/div[2]/div/div[1]/strong/text()').get(default='')
        GS = response.xpath('/html/body/div[3]/div[2]/div[2]/div/div[1]/p[1]/a[1]/text()').get(default='')
        YQ1 = response.xpath('/html/body/div[3]/div[2]/div[3]/div[1]/div//p/text()').extract()
        YQ2 = response.xpath('/html/body/div[3]/div[2]/div[3]/div[1]/div//span/text()').extract()
        YQ3 = response.xpath('/html/body/div[3]/div[2]/div[3]/div[1]/div/text()').extract()
        if len(YQ1) > 1:
            YQ = YQ1

        elif len(YQ2) > 1:
            YQ = YQ2

        elif len(YQ3) > 1:
            YQ = YQ3

        items = {
            "职位:":ZW,
            "工资:": GZ,
            "公司:":GS,
            "要求:":YQ
            }
        yield items
